package cc.togetherit.jcrawler.fetch;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlcleaner.TagNode;

import cc.togetherit.jcrawler.checkRepeat.RepeatCheck;
import cc.togetherit.jcrawler.dataPool.CrawlerPool;
import cc.togetherit.jcrawler.util.HtmlCleanerUtil;

/****
 * XPath FetchEngine
 * 
 * @author jackie
 *
 */
public class DefaultFetchEngine extends AbstractFetchEngine {



    public DefaultFetchEngine(CrawlerPool<String> downPool, CrawlerPool<FetchRequest> fetchPool, CrawlerPool<FetchRequest> parsePool, RepeatCheck check) {
        super(downPool, fetchPool, parsePool, check);
    }

    public FetchResponse fetch(FetchRequest content) {

        TagNode tag = HtmlCleanerUtil.getInstance().getHtmlTag(content.getContent());
        FetchResponse response = new FetchResponse();
        if (tag == null) {
            return response;
        }
        try {
            URL u;
            u = new URL(content.getUrl());
            Object[] obj = tag.evaluateXPath("//a");
            String link;
            for (Object node : obj) {
                link = ((TagNode) node).getAttributeByName("href");
                if (link != null && !link.startsWith("javascript")) { // remove

                    // if(!visited(link) &&shouldFetched(link)){
                    response.addLink(new URL(u, link).toString());
                    // }
                }
            }
        } catch (MalformedURLException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return response;
    }

    @Override
    public boolean visited(String link) {
        return this.getCheck().visited(link);
    }

    public boolean shouldFetched(String url) {

        if (this.getCrawlPatternList() == null || this.getCrawlPatternList().size() == 0) {
            return true;
        }
        Matcher m;
        for (Pattern pattern : getCrawlPatternList()) {
            m = pattern.matcher(url);
            if (m.find()) {
                m = null;
                return true;
            }
            m = null;
        }
        return false;

    }



}
